package at.ac.tuwien.ir.controller;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

public class Tokenizer implements ITokenizer
{
	private static List<String> StopWords = new LinkedList<String>();
	
	public Tokenizer()
	{
		/*StopWords.add("the");
		StopWords.add("it");
		StopWords.add("of");
		StopWords.add("in");*/
	}
	
	public List<String> tokenizeText(StringBuilder text)
	{
		List<String> tokens = new ArrayList<String>();
		
		String[] tokenArray = text.toString().split("[ ,<>:\t]");
		String token;
		
		for(String t : tokenArray) {
			token = t.replaceAll("[-=/\\?*#^%!()+.~#\"${}&'\\|;\\]\\[\f\27\\\\]", "");
			token = token.replace("\27", "");
			if(!token.isEmpty() && token.length() > 1 && !StopWords.contains(token))
				tokens.add(token.toLowerCase());
		}
		
		return tokens;
	}
}
